/**
 * Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

#include <signal.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/file.h>
#include <dirent.h>
#include <sys/types.h>
#include <linux/vm_sockets.h>
#include <glib.h>
#include <poll.h>
#include <sys/file.h>
#include "enclave.h"
#include "virtio_qtbox_sandbox.h"
#include "util.h"
#include <cjson/cJSON.h>

#define QTBOX_DEV              "/dev/qtbox_service0"
#define ENCLAVE_CONFIG_PATH    "/var/run/enclave_%lu.conf"
#define ENCLAVE_CONFIG_FMT    "EnclaveID : %lu\nProcessID : %d\nEnclaveCID" \
                " : %ld\nNumberOfCPUs : %d\nMemoryMiB : %lu\nDebugMode : %d"
#define ENCLAVE_LOG_DEV                 "/dev/sandbox-log-%lu"
#define ENCLAVE_SERV_PATH               "/var/run/enclave/enclave_serv_%lu.sock"
#define ENCLAVE_LISTEN_HEARTBEAT_LOCK_FILE "/var/run/enclave/enclave_vsock.lock"
#define HEART_BEAT                      0xB7
#define HEART_BEAT_MSG_SIZE             1
#define ENCLAVE_DIR                     "/var/run/enclave/"
#define ENCLAVE_DIR_FMT                 "/var/run/enclave/%s"
#define ENCLAVE_ENV_CONFIG_FILE         "/etc/qingtian/enclave/qt-enclave-env.conf"
#define ENCLAVE_TIME_OUT                3000
#define ENCLAVE_WAIT_TIME_US            10000
#define EIF_IMAGE_RATIO                 4
#define CONFIG_LINE_MAX_SIZE            1024
#define ENCLAVE_HUGE_MEM_2MB            2
#define ENCLAVE_HUGE_MEM_1G             1024
#define ENCLAVE_MEM_SHIFT               20
#define ENCLAVE_FLOCK_TIMEOUT           60
#define ENCLAVE_FLOCK_CHECK_RATIO       100
#define ENCLAVE_DEFINE_WAIT_PERIOD      500000 /* 500ms */
#define ENCLAVE_DEFINE_RETRY_TIMES      4

static char config_path[PATH_MAX];

static int sandbox_vm_define(struct enclave_instance_info *info)
{
    int dev_fd;
    int enclave_fd;
    struct vqtbox_sandbox_define define_info = { 0 };
    int retry_times = 0;

    dev_fd = open(QTBOX_DEV, O_RDWR | O_CLOEXEC);
    if (dev_fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: Open, File: %s [%m]", QTBOX_DEV);
        return -1;
    }

    define_info.launch_mode = info->debug ?
            VQTBOX_SANDBOX_VM_DEBUG_MODE : VQTBOX_SANDBOX_VM_QUIET_MODE;
    define_info.image_format = VQTBOX_SANDBOX_VM_EIF_FORMAT;

    while (retry_times++ < ENCLAVE_DEFINE_RETRY_TIMES) {
        enclave_fd = ioctl(dev_fd, VQTBOX_SANDBOX_DEFINE, &define_info);
        if (enclave_fd >=0 || errno != VIRTIO_QTBOX_SANDBOX_VM_ERR_INSTANCE_NOSPC) {
            break;
        }

        log_info("sandbox vm define do retry wait, times : %d", retry_times);
        usleep(ENCLAVE_DEFINE_WAIT_PERIOD);
    }

    if (enclave_fd < 0)
        qt_log_err(QT_IOCTL_DEFINE_SANDBOX_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "create VM failed, [%m], Please ask huawei technical support for help");
    else
        info->enclave_id = define_info.sid;

    close(dev_fd);
    return enclave_fd;
}

static int vm_get_config_hugepage_size(int *size)
{
    char line[CONFIG_LINE_MAX_SIZE];
    FILE *fp;

    fp = fopen(ENCLAVE_ENV_CONFIG_FILE, "r");
    if (!fp) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: Open, File: %s [%m]", ENCLAVE_ENV_CONFIG_FILE);
        return -1;
    }

    while (fgets(line, sizeof(line), fp) != NULL) {
        if (line[0] == '\n')
            continue;
        if (line[0] == '#')
            continue;

        if (sscanf(line, " hugepage_size :%d", size) == 1) {
            if (*size != ENCLAVE_HUGE_MEM_2MB && *size != ENCLAVE_HUGE_MEM_1G) {
                qt_log_err(QT_INVALID_PARS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                           "Invalid hugepage size %d", *size);
                fclose(fp);
                return -1;
            }
            log_info("get hugepage_size %dMb", *size);
            fclose(fp);
            return 0;
        }
    }

    qt_log_err(QT_MISSING_ARGS_IN_CONF_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
               "Can not find hugepage_size config");
    fclose(fp);
    return -1;
}

static int vm_load_image(struct vqtbox_sandbox_mem_range *region,
                         uint64_t offset, char *image)
{
    int image_fd = -1;
    size_t image_size = 0;
    char *enclave_image = NULL;
    struct stat stat_buf = {};
    int rc;

    rc = stat(image, &stat_buf);
    if (rc < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: stat, File: %s [%m]", image);
        return -1;
    }

    image_size = stat_buf.st_size;
    if (region->len < offset) {
        qt_log_err(QT_LOAD_IMAGE_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Image offset %lu over total memory size %lu", offset, region->len);
        return -1;
    }

    if (region->len - offset < image_size) {
        qt_log_err(QT_LOAD_IMAGE_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Memory too smaller to hold image size, len %lu, offset %lu, image size %lu",
                    region->len, offset, image_size);
        return -1;
    }

    image_fd = open(image, O_RDONLY);
    if (image_fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: open, File: %s [%m]", image);
        return -1;
    }

    enclave_image = mmap(NULL, image_size, PROT_READ,
                         MAP_PRIVATE, image_fd, 0);
    if (enclave_image == MAP_FAILED) {
        qt_log_err(QT_LOAD_IMAGE_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Error in mmap enclave image [%m]");
        close(image_fd);
        return -1;
    }

    log_info("load enclave image offset:0x%lx image_size:0x%lx",
             offset, image_size);

    memcpy((void *)region->start + offset, enclave_image, image_size);
    munmap(enclave_image, image_size);
    close(image_fd);
    return 0;
}

static int vm_set_mem_region(int enclave_fd, struct enclave_instance_info *info,
                             struct vqtbox_sandbox_vm_cap *cap)
{
    int rc;
    int hugepage_size;
    int flag;
    void *addr;
    struct vqtbox_sandbox_mem_range region = { 0 };

    rc = vm_get_config_hugepage_size(&hugepage_size);
    if (rc < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Get hugepage size failed\n");
        return -1;
    }

    flag = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB;
    flag |= hugepage_size == ENCLAVE_HUGE_MEM_2MB ? MAP_HUGE_2MB : MAP_HUGE_1GB;

    addr = mmap(NULL, info->memory << ENCLAVE_MEM_SHIFT,
                PROT_READ | PROT_WRITE, flag, -1, 0);
    if (addr == MAP_FAILED) {
        qt_log_err(QT_MMAP_MEM_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Mmap memory failed [%m]");
        return -1;
    }

    region.start = (__u64)addr;
    region.len = info->memory << ENCLAVE_MEM_SHIFT;
    rc = ioctl(enclave_fd, VQTBOX_SANDBOX_ADD_MEM, &region);
    if (rc < 0) {
        munmap(addr, info->memory << ENCLAVE_MEM_SHIFT);
        qt_log_err(QT_MMAP_MEM_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Set user memory region failed [%m],"
                   " Please ask huawei technical support for help");
        return -1;
    }

    rc = vm_load_image(&region, cap->image_offset, info->image);
    if (rc) {
        munmap(addr, info->memory << ENCLAVE_MEM_SHIFT);
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR, "Load image failed");
        return -1;
    }

    log_info("set user mem region 0x%lx[0x%lx]",
             region.start, region.len);
    return 0;
}

static int vm_set_cpu(int enclave_fd, struct enclave_instance_info *info)
{
    int rc;
    int i;
    uint64_t cpu;

    for (i = 0; i < info->cpus; i++) {
        cpu = 0;
        rc = ioctl(enclave_fd, VQTBOX_SANDBOX_ADD_CPU, &cpu);
        if (rc < 0) {
            qt_log_err(QT_IOCTL_ADD_CPU_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                       "Add vCPU failed [%m], Please ask huawei technical support for help");
            return -1;
        }
        log_info("add vCPU %lu to the enclave", cpu);
    }

    return 0;
}

static void process_cmd(int client_fd, struct enclave_instance_info *info)
{
    socklen_t len;
    struct cmd_msg msg;

    memset(&msg, 0, sizeof(msg));
    len = read(client_fd, &msg, sizeof(msg));
    if (len != sizeof(msg)) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "wrong msg, receive %d, expect %lu", len, sizeof(msg));
        return;
    }

    switch (msg.cmd_type) {
        case ENCLAVE_STOP:
            log_info("recieve cmd STOP, enclave id %lu",
                msg.data.enclave_id);
            if (info->enclave_id != msg.data.enclave_id) {
                qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                           "Received enclave id %lu does not match existed enclave id %lu",
                           msg.data.enclave_id,
                           info->enclave_id);
                exit(0);
            }
            len = write(client_fd, &msg, sizeof(msg));
            if (len != sizeof(msg))
                qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                           "Write back failed");
            exit(0);
        case ENCLAVE_QUERY:
            log_info("recieve cmd QUERY");
            memcpy(&msg.data.info, info, sizeof(*info));
            msg.data.info.image = NULL;
            len = write(client_fd, &msg, sizeof(msg));
            if (len != sizeof(msg))
                qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                           "Write back failed");
            break;
        default:
            qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                       "CMD not support");
            break;
    }
}

static int vsock_listen(void)
{
    int vsock_fd = -1;
    int rc = -EINVAL;

    struct sockaddr_vm vsock_addr = {
        .svm_family = AF_VSOCK,
        .svm_cid = DEFAULT_HOST_CID,
        .svm_port = DEFAULT_ENCLAVE_PORT,
    };

    vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0);
    if (vsock_fd < 0) {
        qt_log_err(QT_SOCKET_INIT_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Error in socket [%m]");
        return rc;
    }

    rc = bind(vsock_fd, (struct sockaddr *)&vsock_addr, sizeof(vsock_addr));
    if (rc < 0) {
        qt_log_err(QT_SOCKET_BIND_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Error in bind [%m]");
        close(vsock_fd);
        return rc;
    }

    rc = listen(vsock_fd, 1);
    if (rc < 0) {
        qt_log_err(QT_SOCKET_LISTEN_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Error in listen [%m]");
        close(vsock_fd);
        return rc;
    }

    log_info("vsock listen cid:%d port:%d",
             DEFAULT_HOST_CID, DEFAULT_ENCLAVE_PORT);
    return vsock_fd;
}

static void set_nonblock(int fd)
{
    int f;

    f = fcntl(fd, F_GETFL);
    if (f == -1) {
        log_warn("fcntl failed %d\n", fd);
        return;
    }

    f = fcntl(fd, F_SETFL, f | O_NONBLOCK);
    if (f == -1)
        log_warn("set fd %d nonblock failed\n", fd);

    return;
}

static int wait_accept(int server_fd)
{
    int client_fd;
    int count = ENCLAVE_TIME_OUT;
    struct sockaddr_vm addr = { 0 };
    socklen_t vsock_len = sizeof(addr);

    set_nonblock(server_fd);
    errno = 0;
    while (count--) {
        /* wait 10ms */
        usleep(ENCLAVE_WAIT_TIME_US);
        client_fd = accept(server_fd, (struct sockaddr *)&addr, &vsock_len);
        if (client_fd < 0) {
            if (errno == EAGAIN || errno == EWOULDBLOCK) {
                continue;
            }
            qt_log_err(QT_SOCKET_ACCEPT_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                       "Error in accept [%m]");
            return -1;
        } else {
            return client_fd;
        }
    }

    qt_log_err(QT_WAIT_HEARTBEAT_TIMEOUT_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
               "Maybe the provided memory is not large enough");
    return -1;
}

static int wait_enclave_ready(int server_fd)
{
    int rc;
    int client_fd;
    uint8_t buf[HEART_BEAT_MSG_SIZE];

    client_fd = wait_accept(server_fd);
    if (client_fd < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Start enclave failed");
        return -1;
    }

    memset(buf, 0, sizeof(buf));
    rc = read(client_fd, buf, sizeof(buf));
    if (rc == HEART_BEAT_MSG_SIZE && buf[0] == HEART_BEAT) {
        log_info("get heart beat from enclave");
        rc = write(client_fd, buf, HEART_BEAT_MSG_SIZE);
        if (rc != HEART_BEAT_MSG_SIZE) {
            rc = -1;
            qt_log_err(QT_WRITE_HEARTBEAT_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                       "Write heart beat failed [%m]");
            goto err;
        }
    } else {
        qt_log_err(QT_READ_HEARTBEAT_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Get heart beat failed, rc %d, [%m]", rc);
        rc = -1;
        goto err;
    }

    rc = 0;
err:
    close(client_fd);
    return rc;
}

static int flock_timeout(int fd, int op, const long int timeout)
{
    long int count;

    if (timeout <= 0)
        return flock(fd, op);
    /* set timeout 60s for waiting for lock */
    count = timeout * ENCLAVE_FLOCK_CHECK_RATIO;
    while (count > 0) {
        if (flock(fd, op | LOCK_NB)) {
            /* sleep 10ms */
            usleep(ENCLAVE_WAIT_TIME_US);
            count--;
            continue;
        }
        break;
    }
    return count == 0 ? -1 : 0;
}

static int listen_heartbeat_lock(void)
{
    int lock_fd;
    /* set timeout 60s for waiting for lock */
    const long int timeout = ENCLAVE_FLOCK_TIMEOUT;
    const char *lock_file = ENCLAVE_LISTEN_HEARTBEAT_LOCK_FILE;

    lock_fd = open(lock_file, O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
    if (lock_fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operations: open, File: %s: [%m]\n", lock_file);
        return -1;
    }
    if (flock_timeout(lock_fd, LOCK_EX, timeout)) {
        qt_log_err(QT_LOCK_ACQUIRE_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Lock %s, errno: %d [%m]\n", lock_file, errno);
        close(lock_fd);
        return -1;
    }
    log_info("lock vsock %s.\n", lock_file);
    return lock_fd;
}

static void listen_heartbeat_unlock(int lock_fd)
{
    if (lock_fd >= 0) {
        log_info("unlock vsock %s.\n", ENCLAVE_LISTEN_HEARTBEAT_LOCK_FILE);
        if (flock(lock_fd, LOCK_UN)) {
            qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                       "Unlock vsock failed, errno:%d [%m]\n", errno);
        }
        close(lock_fd);
    }
}

static int vm_start(int enclave_fd, int64_t cid)
{
    int rc;
    int server_fd;
    struct vqtbox_sandbox_start start = { 0 };
    int lock_fd;

    lock_fd = listen_heartbeat_lock();
    if (lock_fd < 0)
        return -1;

    server_fd = vsock_listen();
    if (server_fd < 0) {
        listen_heartbeat_unlock(lock_fd);
        return -1;
    }
    start.cid = cid;
    rc = ioctl(enclave_fd, VQTBOX_SANDBOX_START, &start);
    if (rc < 0) {
        rc = -1;
        qt_log_err(QT_IOCTL_START_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Start enclave failed, %m, Please ask huawei technical support for help");
        goto err;
    }

    if (wait_enclave_ready(server_fd)) {
        rc = -1;
        goto err;
    }

    log_info("enclave start successfully");
    rc = 0;
err:
    listen_heartbeat_unlock(lock_fd);
    close(server_fd);
    return rc;
}

static void remove_enclave_config(uint64_t enclave_id)
{
    char path[PATH_MAX] = {0};

    (void)snprintf(config_path, PATH_MAX, ENCLAVE_CONFIG_PATH, enclave_id);
    unlink(config_path);
    (void)snprintf(path, PATH_MAX, ENCLAVE_SERV_PATH, enclave_id);
    unlink(path);
}

static bool enclave_pid_valid(int pid)
{
    int fd, len;
    char path[PATH_MAX];
    char comm[PATH_MAX] = { 0 };

    if (snprintf(path, PATH_MAX, "/proc/%d/comm", pid) < 0)
        return false;

    fd = open(path, O_RDONLY);
    if (fd < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Enclave pid %d not exist [%m]!", pid);
        return false;
    }

    len = read(fd, comm, PATH_MAX - 1);
    close(fd);
    if (len <= 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Get pid comm failed [%m]!");
        return false;
    }

    if (strncmp(comm, ENCLAVE_BIN, strlen(ENCLAVE_BIN) + 1)) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "process %d comm is %s, not %s", pid, comm, ENCLAVE_BIN);
        return false;
    }

    return true;
}

static int enclave_pid(uint64_t enclave_id)
{
    int fd, len;
    struct enclave_instance_info info;

    if (snprintf(config_path, PATH_MAX, ENCLAVE_CONFIG_PATH, enclave_id) < 0)
        return -1;

    fd = open(config_path, O_RDONLY);
    if (fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operations: open, File: %s, errno: %d [%m]", config_path, errno);
        return -1;
    }

    len = read(fd, &info, sizeof(info));
    close(fd);
    if (len != sizeof(info)) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operations: read, File: %s, read len: %d", config_path, len);
        return -1;
    }

    if (enclave_pid_valid(info.pid)) {
        qt_log_err(QT_INVALID_ENCLAVE_PID_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Please ask huawei techinical support for help");
        return info.pid;
    }

    return -1;
}

static int write_enclave_config(struct enclave_instance_info *info)
{
    int fd;
    char buf[PATH_MAX] = { 0 };

    info->pid = getpid();
    if (snprintf(config_path, PATH_MAX, ENCLAVE_CONFIG_PATH, info->enclave_id) < 0)
        return -1;

    fd = open(config_path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP);
    if (fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Enclave[%lu] create config failed [%m]", info->enclave_id);
        return -1;
    }

    if (write(fd, info, sizeof(*info)) != sizeof(*info)) {
        qt_log_err(QT_WRITE_ENCLAVE_CONFIG_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Enclave[%lu] write config failed, errno: %d [%m]", errno, info->enclave_id);
        close(fd);
        return -1;
    }

    (void)snprintf(buf, PATH_MAX, ENCLAVE_CONFIG_FMT, info->enclave_id, info->pid,
                   info->vsock_cid, info->cpus, info->memory, info->debug);
    log_info("Start enclave:\n%s", buf);
    close(fd);
    return 0;
}

static gboolean handle_accept(GIOChannel *ch,
                              GIOCondition condition,
                              gpointer data)
{
    int client_fd;
    client_fd = serv_accept(g_io_channel_unix_get_fd(ch));
    if (client_fd < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "accept client failed\n");
        return TRUE;
    }

    process_cmd(client_fd, data);
    close(client_fd);
    return TRUE;
}

static gboolean handle_enclave_exit(GIOChannel *ch,
                                    GIOCondition condition,
                                    gpointer data)
{
    GMainLoop *loop = data;
    log_info("get enclave exit event\n");
    g_main_loop_quit(loop);
    return FALSE;
}

static void add_enclave_watch(int fd, GMainLoop *loop)
{
    GIOChannel *ch;
    ch = g_io_channel_unix_new(fd);
    g_io_add_watch(ch, G_IO_HUP, handle_enclave_exit, loop);
}

static void add_cmd_watch(struct enclave_instance_info *info, GMainLoop *loop)
{
    int fd;
    GIOChannel *ch;
    char sock_path[PATH_MAX];

    if (snprintf(sock_path, PATH_MAX, ENCLAVE_SERV_PATH, info->enclave_id) < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Failed to assemble sock_path(err: %d)\n", errno);
        exit(1);
    }
    fd = serv_listen(sock_path);
    if (fd < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "listen command unix socket failed\n");
        exit(1);
    }
    ch = g_io_channel_unix_new(fd);
    if (ch == NULL) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "add io channel to unix socket fd failed\n");
        exit(1);
    }
    g_io_add_watch(ch, G_IO_IN, handle_accept, info);
}

static void start_main_loop(struct enclave_instance_info *info, int enclave_fd)
{
    GMainLoop *loop = g_main_loop_new(NULL, FALSE);

    add_cmd_watch(info, loop);
    add_enclave_watch(enclave_fd, loop);
    g_main_loop_run(loop);
    g_main_loop_unref(loop);
}

static int sandbox_get_cap(struct vqtbox_sandbox_vm_cap *cap)
{
    int dev_fd = -1;
    int ret;

    dev_fd = open(QTBOX_DEV, O_RDWR | O_CLOEXEC);
    if (dev_fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: Open, File: %s", QTBOX_DEV);
        return -1;
    }

    ret = ioctl(dev_fd, VQTBOX_SANDBOX_GET_CAP, cap);
    if (ret != 0) {
        close(dev_fd);
        qt_log_err(QT_IOCTL_GET_SANDBOX_CAP_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "get enclave capability failed %d\n", ret);
        return -1;
    }

    close(dev_fd);
    return 0;
}

static int check_instance_info(struct enclave_instance_info *info, struct vqtbox_sandbox_vm_cap *cap)
{
    int rc;
    size_t min_mem_require = 0;
    struct stat stat_buf = {};

    if (info->vsock_cid < ENCLAVE_CID_MIN || info->vsock_cid > ENCLAVE_CID_MAX) {
        qt_log_err(QT_INVALID_ARGS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Min CID is %d, max CID is %ld", ENCLAVE_CID_MIN, ENCLAVE_CID_MAX);
        return -1;
    }

    if (info->cpus <= 0 || (uint64_t)info->cpus > cap->max_vcpus_per_inst) {
        qt_log_err(QT_INVALID_ARGS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Invalid cpu count %d, check the user guide to ensure that "
                   "the parameters are correct\n", info->cpus);
        return -1;
    }

    if (info->memory < MIN_MEM_MB || info->memory > MAX_MEM_MB) {
        qt_log_err(QT_INVALID_ARGS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Min memory %dMb, max memory %dMb, actual memory %luMb\n",
                   MIN_MEM_MB, MAX_MEM_MB, info->memory);
        return -1;
    }

    rc = stat(info->image, &stat_buf);
    if (rc < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: stat, File: %s [%m]", info->image);
        return -1;
    }

    min_mem_require = (stat_buf.st_size * EIF_IMAGE_RATIO) >> ENCLAVE_MEM_SHIFT;
    if (info->memory < min_mem_require) {
        qt_log_err(QT_INVALID_ARGS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Need at least %luMb memory, which is %d times eif image size,"
                   " actual memory is %lu now", min_mem_require, EIF_IMAGE_RATIO, info->memory);
        return -1;
    }

    return 0;
}

static int print_json_info(cJSON* obj)
{
    char *console_str = NULL;

    console_str = cJSON_Print(obj);
    if (console_str == NULL) {
        qt_log_err(QT_GET_JSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Please check cjson library");
        return -1;
    }

    printf("%s\n", console_str);

    free(console_str);
    return 0;
}

static cJSON* get_enclave_info_json_object(struct enclave_instance_info *info)
{
    cJSON *monitor;

    monitor = cJSON_CreateObject();
    if (monitor == NULL) {
        qt_log_err(QT_CREATE_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Please check cjson library");
        return NULL;
    }

    if (cJSON_AddNumberToObject(monitor, "EnclaveID", info->enclave_id) == NULL) {
        qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add EnclaveID err");
        goto end;
    }

    if (info->pid) {
        if (cJSON_AddNumberToObject(monitor, "ProcessID", info->pid) == NULL) {
            qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                       "Add ProcessID err");
            goto end;
        }
    }

    if (cJSON_AddNumberToObject(monitor, "EnclaveCID", info->vsock_cid) == NULL) {
        qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add EnclaveCID err");
        goto end;
    }

    if (cJSON_AddNumberToObject(monitor, "NumberOfCPUs", info->cpus) == NULL) {
        qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add NumberOfCPUs err");
        goto end;
    }

    if (cJSON_AddNumberToObject(monitor, "MemoryMiB", info->memory) == NULL) {
        qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add MemoryMiB err");
        goto end;
    }

    if (cJSON_AddStringToObject(monitor, "LaunchMode",
                                info->debug ? "debug" : "normal") == NULL) {
        qt_log_err(QT_ADD_STR_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add Enclave Mode err");
        goto end;
    }
    return monitor;

end:
    cJSON_Delete(monitor);
    return NULL;
}

int enclave_run(struct enclave_instance_info *info)
{
    int rc;
    int enclave_fd = -1;
    struct vqtbox_sandbox_vm_cap cap = { 0 };
    cJSON *monitor = NULL;

    rc = sandbox_get_cap(&cap);
    if (rc)
        return -1;

    rc = check_instance_info(info, &cap);
    if (rc)
        return -1;

    enclave_fd = sandbox_vm_define(info);
    if (enclave_fd < 0)
        return -1;

    if (vm_set_mem_region(enclave_fd, info, &cap))
        goto err;

    if (vm_set_cpu(enclave_fd, info))
        goto err;

    if (vm_start(enclave_fd, info->vsock_cid))
        goto err;

    printf("Started enclave with EnclaveID : %lu, EnclaveCID : %ld, " \
           "NumberOfCPUs : %d, MemoryMiB : %lu\n",
           info->enclave_id, info->vsock_cid, info->cpus, info->memory);

    monitor = get_enclave_info_json_object(info);
    if (monitor == NULL)
        goto err;

    rc = print_json_info(monitor);
    cJSON_Delete(monitor);
    if (rc)
        goto err;

    if (daemon(1, 0) < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "daemonize failed");
        goto err;
    }

    rc = write_enclave_config(info);
    if (rc < 0)
        goto err;

    /* Blocks until recieve terminate msg */
    start_main_loop(info, enclave_fd);
    remove_enclave_config(info->enclave_id);
    return 0;

err:
    close(enclave_fd);
    return -1;
}

static int send_message(const char *path, struct cmd_msg *msg)
{
    int ret = -1;
    int fd, len;

    if (path == NULL || msg == NULL)
        return ret;

    log_info("connecting to target enclave server %s", path);
    fd = cli_conn(path);
    if (fd < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "Connect to enclave server failed, %s", path);
        return -1;
    }

    len = write(fd, msg, sizeof(struct cmd_msg));
    if (len != sizeof(struct cmd_msg)) {
        qt_log_err(QT_WRITE_CMD_TO_ENCLAVE_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "write msg failed, len: %d", len);
        goto clean;
    }

    memset(msg, 0, sizeof(struct cmd_msg));
    len = read(fd, msg, sizeof(struct cmd_msg));
    if (len != sizeof(struct cmd_msg)) {
        qt_log_err(QT_READ_INFO_FROM_ENCLAVE_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                   "read failed, len: %d", len);
        goto clean;
    }

    ret = len;
clean:
    close(fd);
    return ret;
}

static int enclave_describe_single_enclave(const char *path, cJSON **enclave_json_obj)
{
    char buf[PATH_MAX];
    struct cmd_msg msg = {0};
    cJSON *monitor = NULL;

    msg.cmd_type = ENCLAVE_QUERY;
    if (send_message(path, &msg) < 0)
        return -1;

    (void)snprintf(buf, PATH_MAX, ENCLAVE_CONFIG_FMT, msg.data.info.enclave_id,
                   msg.data.info.pid, msg.data.info.vsock_cid,
                   msg.data.info.cpus, msg.data.info.memory, msg.data.info.debug);
    log_info("%s", buf);

    monitor = get_enclave_info_json_object(&msg.data.info);
    if (monitor == NULL)
        return -1;

    *enclave_json_obj = monitor;

    return 0;
}

/*
 * we need find a better way to get all enclave_id
*/
static int enclave_describe_all_enclaves()
{
    struct dirent *dir;
    char path[PATH_MAX] = {0};
    int ret;
    DIR *d;
    cJSON *monitor = NULL;
    cJSON *query_arr = NULL;

    d = opendir(ENCLAVE_DIR);
    if (!d) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "File Operation: open, File: %s, errno: %d [%m]", ENCLAVE_DIR, errno);
        return -1;
    }

    query_arr = cJSON_CreateArray();
    if (query_arr == NULL) {
        qt_log_err(QT_CREATE_CJSON_ARR_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Please check cjson library");
        ret = -1;
        goto err;
    }

    while ((dir = readdir(d)) != NULL) {
        if (dir->d_type != DT_SOCK)
            continue;
        if (snprintf(path, PATH_MAX, ENCLAVE_DIR_FMT, dir->d_name) < 0) {
            log_warn("Failed to assemble enclave running path(err: %d)", errno);
            continue;
        }
        ret = enclave_describe_single_enclave(path, &monitor);
        if (ret < 0) {
            log_info("%s may not exist now", path);
            continue;
        }
        cJSON_AddItemToArray(query_arr, monitor);
    }

    /* print enclave info in json */
    ret = print_json_info(query_arr);

    cJSON_Delete(query_arr);
err:
    closedir(d);
    return ret;
}

int enclave_describes(int64_t enclave_id)
{
    char path[PATH_MAX] = {0};
    int ret;
    cJSON *monitor = NULL;

    if (enclave_id >= 0) {
        if (snprintf(path, PATH_MAX, ENCLAVE_SERV_PATH, (uint64_t)enclave_id) < 0)
            return -1;
        if (access(path, F_OK) != 0) {
            qt_log_err(QT_ENCLAVE_NOT_RUNNING_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                       "Enclave %lu may not be running", enclave_id);
            return -1;
        }
        ret = enclave_describe_single_enclave(path, &monitor);
        if (ret < 0) {
            return ret;
        }
        /* print enclave info in json */
        ret = print_json_info(monitor);
        cJSON_Delete(monitor);

        return ret;
    }

    ret = enclave_describe_all_enclaves();
    if (ret < 0) {
        qt_log_err(QT_UNDEINED_ERROR_NUM, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Describe all enclaves failure");
    }
    return ret;
}

int enclave_destroy(uint64_t enclave_id)
{
    int pid;
    char path[PATH_MAX] = {0};
    struct cmd_msg msg = {0};
    cJSON *monitor = NULL;
    int rc;

    if (snprintf(path, PATH_MAX, ENCLAVE_SERV_PATH, enclave_id) < 0)
        return -1;

    if (access(path, F_OK) != 0) {
        qt_log_err(QT_ENCLAVE_NOT_RUNNING_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Enclave %lu may not be running", enclave_id);
        return -1;
    }

    msg.cmd_type = ENCLAVE_STOP;
    msg.data.enclave_id = enclave_id;

    if (send_message(path, &msg) < 0)
        goto force_kill;

    if (msg.data.enclave_id != enclave_id) {
        qt_log_err(QT_READ_INFO_FROM_ENCLAVE_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "not expect enclave_id, expect %lu, recieve %lu", enclave_id, msg.data.enclave_id);
        return -1;
    }

    remove_enclave_config(enclave_id);
    log_info("stop enclave %lu successfully", enclave_id);
    printf("stop enclave %lu successfully, begin to release\n", enclave_id);

    /* print enclave info in json */
    monitor = cJSON_CreateObject();
    if (monitor == NULL) {
        qt_log_err(QT_CREATE_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR, "Please check cjson library");
        return -1;
    }

    if (cJSON_AddNumberToObject(monitor, "EnclaveID", enclave_id) == NULL) {
        qt_log_err(QT_ADD_NUM_TO_CJSON_OBJ_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Add EnclaveID in json object failure");
        cJSON_Delete(monitor);
        return -1;
    }

    rc = print_json_info(monitor);
    cJSON_Delete(monitor);
    return rc;

force_kill:
    pid = enclave_pid(enclave_id);
    if (pid < 0)
        return -1;

    kill(pid, SIGKILL);
    unlink(path);
    remove_enclave_config(enclave_id);
    log_warn("force exiting the enclave %lu!", enclave_id);
    printf("force exiting the enclave %lu!\n", enclave_id);
    return 0;
}

int enclave_console(uint64_t enclave_id)
{
    char enclave_path[PATH_MAX];
    char log_dev[PATH_MAX];
    char buf[1024];
    int fd;
    int ret;
    /* poll a char dev generated by the enclave in debug mode */
    struct pollfd fds[1];
    int len;

    memset(enclave_path, 0, PATH_MAX);
    if (snprintf(enclave_path, PATH_MAX, ENCLAVE_SERV_PATH, enclave_id) < 0)
        return -1;
    if (access(enclave_path, F_OK) != 0) {
        qt_log_err(QT_ENCLAVE_NOT_RUNNING_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Enclave %lu may not be running", enclave_id);
        return -1;
    }

    memset(log_dev, 0, PATH_MAX);
    if (snprintf(log_dev, PATH_MAX, ENCLAVE_LOG_DEV, enclave_id) < 0)
        return -1;
    if (access(log_dev, F_OK) != 0) {
        qt_log_err(QT_NOT_IN_DEBUG_MODE_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Please make sure that the enclave %lu is running in debug mode \n",
                   enclave_id);
        return -1;
    }

    fd = open(log_dev, O_RDONLY);
    if (fd < 0) {
        qt_log_err(QT_FILE_OPS_ERROR, QT_DISPLAY_ADDITIONAL_ERROR,
                   "Operation: Open, File: %s, errno: %d [%m]"
                   "Maybe because someone is using console command on the same enclave. \n",
                   log_dev, errno);
        return -1;
    }

    fds[0].fd = fd;
    fds[0].events = POLLIN | POLLRDNORM;
    while (1) {
        /* poll will be blocked in infinite timeout */
        ret = poll(fds, sizeof(fds) / sizeof(fds[0]), -1);
        if (ret < 0) {
            qt_log_err(QT_CONSOLE_READ_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                       "Polling error, [%m]");
            goto clean;
        } else {
            if (!(fds[0].revents & (POLLIN | POLLRDNORM)))
                break;
            memset(buf, 0, sizeof(buf));
            len = read(fd, buf, sizeof(buf) - 1);
            if (len == 0) {
                log_info("Enclave console finishes !");
                break;
            } else if (len < 0) {
                qt_log_err(QT_CONSOLE_READ_ERROR, QT_NOT_DISPLAY_ADDITIONAL_ERROR,
                           "Enclave console read err, errno: %d [%m]", errno);
                ret = -1;
                goto clean;
            } else {
                printf("%s", buf);
            }
        }
    }
    ret = 0;

clean:
    close(fd);
    return ret;
}
